import logging
import time

from scrapy.dupefilters import BaseDupeFilter
from scrapy.utils.request import request_fingerprint

from . import defaults
from .connection import get_redis_from_settings

from Spider.util.bloomfilterOnRedis import BloomFilter

logger = logging.getLogger(__name__)


"""
由Scheduler模块进行调度,负责执行requst的去重并保存到redis中,作为下一个判断request是否重复的依据
"""


# TODO: Rename class to RedisDupeFilter.
class RFPDupeFilter(BaseDupeFilter):
    logger = logger

    def __init__(self, server, key, debug=False):
        self.server = server
        self.key = key
        self.debug = debug
        self.logdupes = True
        self.bf = BloomFilter(conn=server, key=key, blockNum=1)

    @classmethod
    def from_settings(cls, settings):
        server = get_redis_from_settings(settings)
        # TODO: Use SCRAPY_JOB env as default and fallback to timestamp.
        key = defaults.DUPEFILTER_KEY % {'timestamp': int(time.time())}
        debug = settings.getbool('DUPEFILTER_DEBUG')
        return cls(server, key=key, debug=debug)

    @classmethod
    def from_crawler(cls, crawler):
        return cls.from_settings(crawler.settings)

    def request_seen(self, request):
        """
        判断当前request是否存在:
            存在返回True
            不存在返回False并存储到redis中
        """
        fp = self.request_fingerprint(request)
        # added = self.server.sadd(self.key, fp)
        # return added == 0
        #BloomFilter
        if self.bf.isContains(fp):
            return True
        else:
            self.bf.insert(fp)
            return False

    def request_fingerprint(self, request):
        """
        通过hash来判断两个url是否相同（相同的url会生成相同的hash结果），
        但是当两个url的地址相同，get型参数相同但是顺序不同时，也会生成相同的hash结果
        """
        return request_fingerprint(request)

    def close(self, reason=''):
        """Delete data on close. Called by Scrapy's scheduler.

        Parameters
        ----------
        reason : str, optional

        """
        self.clear()

    def clear(self):
        """Clears fingerprints data."""
        self.server.delete(self.key)

    def log(self, request, spider):
        """Logs given request.

        Parameters
        ----------
        request : scrapy.http.Request
        spider : scrapy.spiders.Spider

        """
        if self.debug:
            msg = "Filtered duplicate request: %(request)s"
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request %(request)s"
                   " - no more duplicates will be shown"
                   " (see DUPEFILTER_DEBUG to show all duplicates)")
            self.logger.debug(msg, {'request': request}, extra={'spider': spider})
            self.logdupes = False
